<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Heap: Sizing and Swapping | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="deploy.html" title="Production Deployment">
<link rel="prev" href="_don_8217_t_touch_these_settings.html" title="Don’t Touch These Settings!">
<link rel="next" href="_file_descriptors_and_mmap.html" title="File Descriptors and MMap">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="administration.html">Administration, Monitoring, and Deployment</a></span>
»
<span class="breadcrumb-link"><a href="deploy.html">Production Deployment</a></span>
»
<span class="breadcrumb-node">Heap: Sizing and Swapping</span>
</div>
<div class="navheader">
<span class="prev">
<a href="_don_8217_t_touch_these_settings.html">« Don’t Touch These Settings!</a>
</span>
<span class="next">
<a href="_file_descriptors_and_mmap.html">File Descriptors and MMap »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="heap-sizing"></a>Heap: Sizing and Swapping<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/510_Deployment/50_heap.asciidoc">edit</a>
</h2>
</div></div></div>
<p>The default installation of Elasticsearch is configured with a 1 GB heap.  For
just about every deployment, this number is usually too small.  If you are using the
default heap values, your cluster is probably configured incorrectly.</p>
<p>There are two ways to change the heap size in Elasticsearch.  The easiest is to
set an environment variable called <code class="literal">ES_HEAP_SIZE</code>.  When the server process
starts, it will read this environment variable and set the heap accordingly.
As an example, you can set it via the command line as follows:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">export ES_HEAP_SIZE=10g</pre>
</div>
<p>Alternatively, you can pass in the heap size via JVM flags when starting
the process, if that is easier for your setup:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">ES_JAVA_OPTS="-Xms10g -Xmx10g" ./bin/elasticsearch <a id="CO302-1"></a><i class="conum" data-value="1"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO302-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Ensure that the min (<code class="literal">Xms</code>) and max (<code class="literal">Xmx</code>) sizes are the same to prevent
the heap from resizing at runtime, a very costly process.</p>
</td>
</tr>
</table>
</div>
<p>Generally, setting the <code class="literal">ES_HEAP_SIZE</code> environment variable is preferred over setting
explicit <code class="literal">-Xmx</code> and <code class="literal">-Xms</code> values.</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_give_less_than_half_your_memory_to_lucene"></a>Give (less than) Half Your Memory to Lucene<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/510_Deployment/50_heap.asciidoc">edit</a>
</h3>
</div></div></div>
<p>A common problem is configuring a heap that is <em>too</em> large.  You have a 64 GB
machine—​and by golly, you want to give Elasticsearch all 64 GB of memory.  More
is better!</p>
<p>Heap is definitely important to Elasticsearch.  It is used by many in-memory data
structures to provide fast operation.  But with that said, there is another major
user of memory that is <em>off heap</em>: Lucene.</p>
<p>Lucene is designed to leverage the underlying OS for caching in-memory data structures.
Lucene segments are stored in individual files.  Because segments are immutable,
these files never change.  This makes them very cache friendly, and the underlying
OS will happily keep hot segments resident in memory for faster access.  These segments
include both the inverted index (for fulltext search) and doc values (for aggregations).</p>
<p>Lucene’s performance relies on this interaction with the OS.  But if you give all
available memory to Elasticsearch’s heap, there won’t be any left over for Lucene.
This can seriously impact the performance.</p>
<p>The standard recommendation is to give 50% of the available memory to Elasticsearch
heap, while leaving the other 50% free.  It won’t go unused; Lucene will happily
gobble up whatever is left over.</p>
<p>If you are not aggregating on analyzed string fields (e.g. you won’t be needing
<a class="xref" href="aggregations-and-analysis.html" title="Aggregations and Analysis">fielddata</a>) you can consider lowering the heap even
more.  The smaller you can make the heap, the better performance you can expect
from both Elasticsearch (faster GCs) and Lucene (more memory for caching).</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="compressed_oops"></a>Don’t Cross 32 GB!<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/510_Deployment/50_heap.asciidoc">edit</a>
</h3>
</div></div></div>
<p>There is another reason to not allocate enormous heaps to Elasticsearch. As it turns
out, the HotSpot JVM uses a trick to compress object pointers when heaps are less
than around 32 GB.</p>
<p>In Java, all objects are allocated on the heap and referenced by a pointer.
Ordinary object pointers (OOP) point at these objects, and are traditionally
the size of the CPU’s native <em>word</em>: either 32 bits or 64 bits, depending on the
processor.  The pointer references the exact byte location of the value.</p>
<p>For 32-bit systems, this means the maximum heap size is 4 GB.  For 64-bit systems,
the heap size can get much larger, but the overhead of 64-bit pointers means there
is more wasted space simply because the pointer is larger.  And worse than wasted
space, the larger pointers eat up more bandwidth when moving values between
main memory and various caches (LLC, L1, and so forth).</p>
<p>Java uses a trick called <a href="https://wikis.oracle.com/display/HotSpotInternals/CompressedOops" class="ulink" target="_top">compressed oops</a>
to get around this problem.  Instead of pointing at exact byte locations in
memory, the pointers reference <em>object offsets</em>.  This means a 32-bit pointer can
reference four billion <em>objects</em>, rather than four billion bytes.  Ultimately, this
means the heap can grow to around 32 GB of physical size while still using a 32-bit
pointer.</p>
<p>Once you cross that magical ~32 GB boundary, the pointers switch back to
ordinary object pointers.  The size of each pointer grows, more CPU-memory
bandwidth is used, and you effectively lose memory.  In fact, it takes until around
40–50 GB of allocated heap before you have the same <em>effective</em> memory of a
heap just under 32 GB using compressed oops.</p>
<p>The moral of the story is this: even when you have memory to spare, try to avoid
crossing the 32 GB heap boundary.  It wastes memory, reduces CPU performance, and
makes the GC struggle with large heaps.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_just_how_far_under_32gb_should_i_set_the_jvm"></a>Just how far under 32gb should I set the JVM?<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/510_Deployment/50_heap.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Unfortunately, that depends.  The exact cutoff varies by JVMs and platforms.
If you want to play it safe, setting the heap to <code class="literal">31gb</code> is likely safe.
Alternatively, you can verify the cutoff point for the HotSpot JVM by adding
<code class="literal">-XX:+PrintFlagsFinal</code> to your JVM options and checking that the value of the
UseCompressedOops flag is true. This will let you find the exact cutoff for your
platform and JVM.</p>
<p>For example, here we test a Java 1.7 installation on MacOSX and see the max heap
size is around 32600mb (~31.83gb) before compressed pointers are disabled:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">$ JAVA_HOME=`/usr/libexec/java_home -v 1.7` java -Xmx32600m -XX:+PrintFlagsFinal 2&gt; /dev/null | grep UseCompressedOops
     bool UseCompressedOops   := true
$ JAVA_HOME=`/usr/libexec/java_home -v 1.7` java -Xmx32766m -XX:+PrintFlagsFinal 2&gt; /dev/null | grep UseCompressedOops
     bool UseCompressedOops   = false</pre>
</div>
<p>In contrast, a Java 1.8 installation on the same machine has a max heap size
around 32766mb (~31.99gb):</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">$ JAVA_HOME=`/usr/libexec/java_home -v 1.8` java -Xmx32766m -XX:+PrintFlagsFinal 2&gt; /dev/null | grep UseCompressedOops
     bool UseCompressedOops   := true
$ JAVA_HOME=`/usr/libexec/java_home -v 1.8` java -Xmx32767m -XX:+PrintFlagsFinal 2&gt; /dev/null | grep UseCompressedOops
     bool UseCompressedOops   = false</pre>
</div>
<p>The moral of the story is that the exact cutoff to leverage compressed oops
varies from JVM to JVM, so take caution when taking examples from elsewhere and
be sure to check your system with your configuration and JVM.</p>
<p>Beginning with Elasticsearch v2.2.0, the startup log will actually tell you if your
JVM is using compressed OOPs or not.  You’ll see a log message like:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">[2015-12-16 13:53:33,417][INFO ][env] [Illyana Rasputin] heap size [989.8mb], compressed ordinary object pointers [true]</pre>
</div>
<p>Which indicates that compressed object pointers are being used.  If they are not,
the message will say <code class="literal">[false]</code>.</p>
<div class="sidebar pagebreak-before">
<div class="titlepage"><div><div>
<p class="title"><strong>I Have a Machine with 1 TB RAM!</strong></p>
</div></div></div>
<p>The 32 GB line is fairly important.  So what do you do when your machine has a lot
of memory?  It is becoming increasingly common to see super-servers with 512–768 GB
of RAM.</p>
<p>First, we would recommend avoiding such large machines (see <a class="xref" href="hardware.html" title="Hardware">Hardware</a>).</p>
<p>But if you already have the machines, you have two practical options:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Are you doing mostly full-text search?  Consider giving 4-32 GB to Elasticsearch
and letting Lucene use the rest of memory via the OS filesystem cache.  All that
memory will cache segments and lead to blisteringly fast full-text search.
</li>
<li class="listitem">
Are you doing a lot of sorting/aggregations?  Are most of your aggregations on numerics,
dates, geo_points and <code class="literal">not_analyzed</code> strings?  You’re in luck!  Give Elasticsearch
somewhere from 4-32 GB of memory and leave the rest for the OS to cache doc values
in memory.
</li>
<li class="listitem">
<p>Are you doing a lot of sorting/aggregations on analyzed strings (e.g. for word-tags,
or SigTerms, etc)?  Unfortunately that means you’ll need fielddata, which means you
need heap space.  Instead of one node with more than 512 GB of RAM, consider running two or
more nodes on a single machine.  Still adhere to the 50% rule, though.  So if your
machine has 128 GB of RAM, run two nodes, each with just under 32 GB.  This means that less
than 64 GB will be used for heaps, and more than 64 GB will be left over for Lucene.</p>
<p>If you choose this option, set <code class="literal">cluster.routing.allocation.same_shard.host: true</code>
in your config.  This will prevent a primary and a replica shard from colocating
to the same physical machine (since this would remove the benefits of replica high availability).</p>
</li>
</ul>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_swapping_is_the_death_of_performance"></a>Swapping Is the Death of Performance<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/510_Deployment/50_heap.asciidoc">edit</a>
</h3>
</div></div></div>
<p>It should be obvious, but it bears spelling out clearly: swapping main memory
to disk will <em>crush</em> server performance.  Think about it: an in-memory operation
is one that needs to execute quickly.</p>
<p>If memory swaps to disk, a 100-microsecond operation becomes one that take 10
milliseconds.  Now repeat that increase in latency for all other 10us operations.
It isn’t difficult to see why swapping is terrible for performance.</p>
<p>The best thing to do is disable swap completely on your system.  This can be done
temporarily:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">sudo swapoff -a</pre>
</div>
<p>To disable it permanently, you’ll likely need to edit your <code class="literal">/etc/fstab</code>.  Consult
the documentation for your OS.</p>
<p>If disabling swap completely is not an option, you can try to lower <code class="literal">swappiness</code>.
This value controls how aggressively the OS tries to swap memory.
This prevents swapping under normal circumstances, but still allows the OS to swap
under emergency memory situations.</p>
<p>For most Linux systems, this is configured using the <code class="literal">sysctl</code> value:</p>
<div class="pre_wrapper lang-bash">
<pre class="programlisting prettyprint lang-bash">vm.swappiness=1 <a id="CO303-1"></a><i class="conum" data-value="1"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO303-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>A <code class="literal">swappiness</code> of <code class="literal">1</code> is better than <code class="literal">0</code>, since on some kernel versions a <code class="literal">swappiness</code>
of <code class="literal">0</code> can invoke the OOM-killer.</p>
</td>
</tr>
</table>
</div>
<p>Finally, if neither approach is possible, you should enable <code class="literal">mlockall</code>.
 file.  This allows the JVM to lock its memory and prevent
it from being swapped by the OS.  In your <code class="literal">elasticsearch.yml</code>, set this:</p>
<div class="pre_wrapper lang-yaml">
<pre class="programlisting prettyprint lang-yaml">bootstrap.mlockall: true</pre>
</div>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="_don_8217_t_touch_these_settings.html">« Don’t Touch These Settings!</a>
</span>
<span class="next">
<a href="_file_descriptors_and_mmap.html">File Descriptors and MMap »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
